diff --git a/swh/web/ui/backend.py b/swh/web/ui/backend.py --- a/swh/web/ui/backend.py +++ b/swh/web/ui/backend.py @@ -230,6 +230,15 @@ return main.storage().stat_counters() +def stat_origin_visits(origin_id): + """Return the dates at which the given origin was scanned for content. + + Returns: + An array of dates + """ + return main.storage().origin_visit_get(origin_id) + + def revision_get_by(origin_id, branch_name, timestamp): """Return occurrence information matching the criterions origin_id, branch_name, ts. diff --git a/swh/web/ui/service.py b/swh/web/ui/service.py --- a/swh/web/ui/service.py +++ b/swh/web/ui/service.py @@ -567,6 +567,17 @@ return backend.stat_counters() +def stat_origin_visits(origin_id): + """Return the dates at which the given origin was scanned for content. + + Returns: + An array of dates in the datetime format + """ + for visit in backend.stat_origin_visits(origin_id): + visit['date'] = visit['date'].timestamp() + yield(visit) + + def lookup_entity_by_uuid(uuid): """Return the entity's hierarchy from its uuid. diff --git a/swh/web/ui/static/js/calendar.js b/swh/web/ui/static/js/calendar.js new file mode 100644 --- /dev/null +++ b/swh/web/ui/static/js/calendar.js @@ -0,0 +1,373 @@ +/** + * Calendar: + * A one-off object that makes an AJAX call to the API's visit stats + * endpoint, then displays these statistics in a zoomable timeline-like + * format. + * Args: + * browse_url: the relative URL for browsing a revision via the web ui, + * accurate up to the origin + * visit_url: the complete relative URL for getting the origin's visit + * stats + * origin_id: the origin being browsed + * zoomw: the element that should contain the zoomable part of the calendar + * staticw: the element that should contain the static part of the calendar + * reset: the element that should reset the zoom level on click + */ + +var Calendar = function(browse_url, visit_url, origin_id, + zoomw, staticw, reset) { + + /** Constants **/ + this.month_names = ['Jan', 'Feb', 'Mar', + 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', + 'Oct', 'Nov', 'Dec']; + + /** Display **/ + this.desiredPxWidth = 7; + this.padding = 0.01; + + /** Object vars **/ + this.origin_id = origin_id; + this.zoomw = zoomw; + this.staticw = staticw; + /** Calendar data **/ + this.cal_data = null; + this.static = { + group_factor: 3600 * 1000, + group_data: null, + plot_data: null + }; + this.zoom = { + group_factor: 3600 * 1000, + group_data: null, + plot_data: null + }; + + /** + * Keep a reference to the Calendar object context. + * Otherwise, 'this' changes to represent current function caller scope + */ + var self = this; + + /** Start AJAX call **/ + $.ajax({ + type: 'GET', + url: visit_url, + success: function(data) { + self.calendar(data); + } + }); + + /** + * Group the plot's base data according to the grouping ratio and the + * range required + * Args: + * groupFactor: the amount the data should be grouped by + * range: + * Returns: + * A dictionary containing timestamps divided by the grouping ratio as + * keys, a list of the corresponding complete timestamps as values + */ + + this.dataGroupedRange = function(groupFactor, range) { + var group_dict = {}; + var start = range.xaxis.from; + var end = range.xaxis.to; + var range_data = self.cal_data.filter(function(item, index, arr) { + return item >= start && item <= end; + }); + for (var date_idx in range_data) { + var date = range_data[date_idx]; + var floor = Math.floor(date / groupFactor); + if (group_dict[floor] == undefined) + group_dict[floor] = [date]; + else + group_dict[floor].push(date); + } + return group_dict; + }; + + /** + * Update the ratio that governs how the data is grouped based on changes + * in the data range or the display size, and regroup the plot's data + * according to this value. + * + * Args: + * element: the element in which the plot is displayed + * plotprops: the properties corresponding to that plot + * range: the range of the data displayed + */ + this.updateGroupFactorAndData = function(element, plotprops, range) { + var milli_length = range.xaxis.to - range.xaxis.from; + var px_length = element.width(); + plotprops.group_factor = Math.floor( + self.desiredPxWidth * (milli_length / px_length)); + plotprops.group_data = self.dataGroupedRange( + plotprops.group_factor, range); + }; + + + /** Get plot data from the group data **/ + this.getPlotData = function(grouped_data) { + var plot_data = []; + if (self.cal_data.length == 1) { + plot_data = [[self.cal_data[0] - 3600*1000*24*30, 0], + [self.cal_data[0], 1], + [self.cal_data[0] + 3600*1000*24*30, 0]]; + } + else { + $.each(grouped_data, function(key, value) { + plot_data.push([value[0], value.length]); + }); + } + return [{ label: 'Calendar', data: plot_data }]; + }; + + this.plotZoom = function(zoom_options) { + return $.plot(self.zoomw, self.zoom.plot_data, zoom_options); + }; + + this.plotStatic = function(static_options) { + return $.plot(self.staticw, self.static.plot_data, static_options); + }; + + /** + * Display a zoomable calendar with click-through links to revisions + * of the same origin + * + * Args: + * data: the data that the calendar should present, as a list of + * POSIX second-since-epoch timestamps + */ + this.calendar = function(data) { + // POSIX timestamps to JS timestamps + self.cal_data = data.map(function(e) + { return Math.floor(e * 1000); }); + /** Bootstrap the group ratio **/ + var cal_data_range = null; + if (self.cal_data.length == 1) { + var padding_qty = 3600*1000*24*30; + cal_data_range = {xaxis: {from: self.cal_data[0] - padding_qty, + to: self.cal_data[0] + padding_qty}}; + } + else + cal_data_range = {xaxis: {from: self.cal_data[0], + to: self.cal_data[self.cal_data.length -1] + } + }; + self.updateGroupFactorAndData(self.zoomw, + self.zoom, + cal_data_range); + self.updateGroupFactorAndData(self.staticw, + self.static, + cal_data_range); + /** Bootstrap the plot data **/ + self.zoom.plot_data = self.getPlotData(self.zoom.group_data); + self.static.plot_data = self.getPlotData(self.zoom.group_data); + + /** + * Return the flot-required function for displaying tooltips, according to + * the group we want to display the tooltip for + * Args: + * group_options: the group we want to display the tooltip for (self.static + * or self.zoom) + */ + function tooltip_fn(group_options) { + return function (label, x_timestamp, y_hits, item) { + var floor_index = Math.floor( + item.datapoint[0] / group_options.group_factor); + var tooltip_text = group_options.group_data[floor_index].map( + function(elem) { + var date = new Date(elem); + var year = (date.getYear() + 1900).toString(); + var month = self.month_names[date.getMonth()]; + var day = date.getDate(); + var hr = date.getHours(); + var min = date.getMinutes(); + if (min < 10) min = '0'+min; + return [day, + month, + year + ',', + hr+':'+min, + 'UTC'].join(' '); + } + ); + return tooltip_text.join('
'); + }; + } + + /** Plot options for both graph windows **/ + var zoom_options = { + legend: { + show: false + }, + series: { + clickable: true, + bars: { + show: true, + lineWidth: 1, + barWidth: self.zoom.group_factor + } + }, + xaxis: { + mode: 'time', + minTickSize: [1, 'day'], + // monthNames: self.month_names, + position: 'top' + }, + yaxis: { + show: false + }, + selection: { + mode: 'x' + }, + grid: { + clickable: true, + hoverable: true + }, + tooltip: { + show: true, + content: tooltip_fn(self.zoom) + } + }; + + var overview_options = { + legend: { + show: false + }, + series: { + clickable: true, + bars: { + show: true, + lineWidth: 1, + barWidth: self.static.group_factor + }, + shadowSize: 0 + }, + yaxis: { + show: false + }, + xaxis: { + mode: 'time', + minTickSize: [1, 'day'] + }, + grid: { + clickable: true, + hoverable: true, + color: '#999' + }, + selection: { + mode: 'x' + }, + tooltip: { + show: true, + content: tooltip_fn(self.static) + } + }; + + function addPadding(options, range) { + var len = range.xaxis.to - range.xaxis.from; + return $.extend(true, {}, options, { + xaxis: { + min: range.xaxis.from - (self.padding * len), + max: range.xaxis.to + (self.padding * len) + } + }); + } + + /** draw the windows **/ + var plot = self.plotZoom(addPadding(zoom_options, cal_data_range)); + var overview = self.plotStatic( + addPadding(overview_options, cal_data_range)); + + var current_ranges = $.extend(true, {}, cal_data_range); + + /** + * Zoom to the mouse-selected range in the given window + * + * Args: + * plotzone: the jQuery-selected element the zoomed plot should be + * in (usually the same as the original 'zoom plot' element) + * range: the data range as a dict {xaxis: {from:, to:}, + * yaxis:{from:, to:}} + */ + function zoom(ranges) { + current_ranges.xaxis.from = ranges.xaxis.from; + current_ranges.xaxis.to = ranges.xaxis.to; + self.updateGroupFactorAndData( + self.zoomw, self.zoom, current_ranges); + self.zoom.plot_data = self.getPlotData(self.zoom.group_data); + var zoomedopts = $.extend(true, {}, zoom_options, { + xaxis: { min: ranges.xaxis.from, max: ranges.xaxis.to }, + series: { + bars: {barWidth: self.zoom.group_factor} + } + }); + return self.plotZoom(zoomedopts); + } + + function resetZoomW(plot_options) { + self.zoom.group_data = self.static.group_data; + self.zoom.plot_data = self.static.plot_data; + self.updateGroupFactorAndData(zoomw, self.zoom, cal_data_range); + plot = self.plotZoom(addPadding(plot_options, cal_data_range)); + } + + // now connect the two + self.zoomw.bind('plotselected', function (event, ranges) { + // clamp the zooming to prevent eternal zoom + if (ranges.xaxis.to - ranges.xaxis.from < 0.00001) + ranges.xaxis.to = ranges.xaxis.from + 0.00001; + // do the zooming + plot = zoom(ranges); + // don't fire event on the overview to prevent eternal loop + overview.setSelection(ranges, true); + }); + + self.staticw.bind('plotselected', function (event, ranges) { + plot.setSelection(ranges); + }); + + function unbindClick() { + self.zoomw.unbind('plotclick'); + self.staticw.unbind('plotclick'); + } + + function bindClick() { + self.zoomw.bind('plotclick', redirect_to_revision); + self.staticw.bind('plotclick', redirect_to_revision); + } + + function redirect_to_revision(event, pos, item) { + if (item) { + var ts = Math.floor(item.datapoint[0] / 1000); // POSIX ts + var url = browse_url + 'ts/' + ts + '/'; + window.location.href = url; + } + } + + reset.click(function(event) { + plot.clearSelection(); + overview.clearSelection(); + current_ranges = $.extend(true, {}, cal_data_range); + resetZoomW(zoom_options); + }); + + $(window).resize(function(event) { + /** Update zoom display **/ + self.updateGroupFactorAndData(zoomw, self.zoom, current_ranges); + self.zoom.plot_data = self.getPlotData(self.zoom.group_data); + /** Update static display **/ + self.updateGroupFactorAndData(staticw, self.static, cal_data_range); + self.static.plot_data = self.getPlotData(self.static.group_data); + /** Replot **/ + plot = self.plotZoom( + addPadding(zoom_options, current_ranges)); + overview = self.plotStatic( + addPadding(overview_options, cal_data_range)); + }); + + bindClick(); + }; +}; diff --git a/swh/web/ui/static/lib/jquery.flot.js b/swh/web/ui/static/lib/jquery.flot.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/jquery.flot.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-flot/jquery.flot.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/jquery.flot.selection.js b/swh/web/ui/static/lib/jquery.flot.selection.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/jquery.flot.selection.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-flot/jquery.flot.selection.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/jquery.flot.time.js b/swh/web/ui/static/lib/jquery.flot.time.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/jquery.flot.time.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-flot/jquery.flot.time.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/jquery.flot.tooltip.js b/swh/web/ui/static/lib/jquery.flot.tooltip.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/jquery.flot.tooltip.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-flot/jquery.flot.tooltip.js \ No newline at end of file diff --git a/swh/web/ui/static/lib/jquery.js b/swh/web/ui/static/lib/jquery.js new file mode 120000 --- /dev/null +++ b/swh/web/ui/static/lib/jquery.js @@ -0,0 +1 @@ +/usr/share/javascript/jquery-flot/jquery.js \ No newline at end of file diff --git a/swh/web/ui/templates/origin.html b/swh/web/ui/templates/origin.html --- a/swh/web/ui/templates/origin.html +++ b/swh/web/ui/templates/origin.html @@ -7,7 +7,20 @@ {% endif %} {% if origin is not none %} + + + + + +
Details on origin {{ origin_id }}: +
+
+
+
+
+
+ {% for key in ['type', 'lister', 'projet', 'url'] %} {% if origin[key] is not none %}
@@ -20,8 +33,9 @@
(some decoding errors)
- {% endif %} + {% endif %}
+ {% endif %} {% endblock %} diff --git a/swh/web/ui/tests/test_backend.py b/swh/web/ui/tests/test_backend.py --- a/swh/web/ui/tests/test_backend.py +++ b/swh/web/ui/tests/test_backend.py @@ -624,6 +624,42 @@ self.storage.stat_counters.assert_called_with() @istest + def stat_origin_visits(self): + # given + expected_dates = [ + { + 'date': datetime.datetime( + 2015, 1, 1, 22, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 1 + }, + { + 'date': datetime.datetime( + 2013, 7, 1, 20, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 2 + }, + { + 'date': datetime.datetime( + 2015, 1, 1, 21, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 3 + } + ] + self.storage.origin_visit_get = MagicMock(return_value=expected_dates) + + # when + actual_dates = backend.stat_origin_visits(5) + + # then + self.assertEqual(actual_dates, expected_dates) + + self.storage.origin_visit_get.assert_called_with(5) + + @istest def directory_entry_get_by_path(self): # given stub_dir_entry = {'id': b'dir-id', diff --git a/swh/web/ui/tests/test_service.py b/swh/web/ui/tests/test_service.py --- a/swh/web/ui/tests/test_service.py +++ b/swh/web/ui/tests/test_service.py @@ -208,6 +208,68 @@ mock_backend.stat_counters.assert_called_with() @patch('swh.web.ui.service.backend') + @istest + def stat_origin_visits(self, mock_backend): + # given + stub_result = [ + { + 'date': datetime.datetime( + 2015, 1, 1, 22, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 1 + }, + { + 'date': datetime.datetime( + 2013, 7, 1, 20, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 2 + }, + { + 'date': datetime.datetime( + 2015, 1, 1, 21, 0, 0, + tzinfo=datetime.timezone.utc), + 'origin': 1, + 'visit': 3 + } + ] + mock_backend.stat_origin_visits.return_value = stub_result + + # when + expected_dates = [ + { + 'date': datetime.datetime( + 2015, 1, 1, 22, 0, 0, + tzinfo=datetime.timezone.utc).timestamp(), + 'origin': 1, + 'visit': 1 + }, + { + 'date': datetime.datetime( + 2013, 7, 1, 20, 0, 0, + tzinfo=datetime.timezone.utc).timestamp(), + 'origin': 1, + 'visit': 2 + }, + { + 'date': datetime.datetime( + 2015, 1, 1, 21, 0, 0, + tzinfo=datetime.timezone.utc).timestamp(), + 'origin': 1, + 'visit': 3 + } + ] + + actual_dates = service.stat_origin_visits(6) + + # then + self.assertEqual(expected_dates, + list(actual_dates)) + + mock_backend.stat_origin_visits.assert_called_once_with(6) + + @patch('swh.web.ui.service.backend') @patch('swh.web.ui.service.hashutil') @istest def hash_and_search(self, mock_hashutil, mock_backend): diff --git a/swh/web/ui/tests/views/test_api.py b/swh/web/ui/tests/views/test_api.py --- a/swh/web/ui/tests/views/test_api.py +++ b/swh/web/ui/tests/views/test_api.py @@ -439,6 +439,93 @@ mock_service.stat_counters.assert_called_once_with() @patch('swh.web.ui.views.api.service') + @istest + def api_1_stat_origin_visits_raise_error(self, mock_service): + # given + mock_service.stat_origin_visits.side_effect = ValueError( + 'voluntary error to check the bad request middleware.') + # when + rv = self.app.get('/api/1/stat/visits/2/') + # then + self.assertEquals(rv.status_code, 400) + self.assertEquals(rv.mimetype, 'application/json') + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, { + 'error': 'voluntary error to check the bad request middleware.'}) + + @patch('swh.web.ui.views.api.service') + @istest + def api_1_stat_origin_visits_raise_swh_storage_error_db( + self, mock_service): + # given + mock_service.stat_origin_visits.side_effect = StorageDBError( + 'SWH Storage exploded! Will be back online shortly!') + # when + rv = self.app.get('/api/1/stat/visits/2/') + # then + self.assertEquals(rv.status_code, 503) + self.assertEquals(rv.mimetype, 'application/json') + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, { + 'error': + 'An unexpected error occurred in the backend: ' + 'SWH Storage exploded! Will be back online shortly!'}) + + @patch('swh.web.ui.views.api.service') + @istest + def api_1_stat_origin_visits_raise_swh_storage_error_api( + self, mock_service): + # given + mock_service.stat_origin_visits.side_effect = StorageAPIError( + 'SWH Storage API dropped dead! Will resurrect from its ashes asap!' + ) + # when + rv = self.app.get('/api/1/stat/visits/2/') + # then + self.assertEquals(rv.status_code, 503) + self.assertEquals(rv.mimetype, 'application/json') + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, { + 'error': + 'An unexpected error occurred in the api backend: ' + 'SWH Storage API dropped dead! Will resurrect from its ashes asap!' + }) + + @patch('swh.web.ui.views.api.service') + @istest + def api_1_stat_origin_visits(self, mock_service): + # given + stub_stats = [ + { + 'date': 1420149600.0, + 'origin': 1, + 'visit': 1 + }, + { + 'date': 1104616800.0, + 'origin': 1, + 'visit': 2 + }, + { + 'date': 1293919200.0, + 'origin': 1, + 'visit': 3 + } + ] + expected_stats = [1104616800.0, 1293919200.0, 1420149600.0] + mock_service.stat_origin_visits.return_value = stub_stats + + # when + rv = self.app.get('/api/1/stat/visits/2/') + + self.assertEquals(rv.status_code, 200) + self.assertEquals(rv.mimetype, 'application/json') + response_data = json.loads(rv.data.decode('utf-8')) + self.assertEquals(response_data, expected_stats) + + mock_service.stat_origin_visits.assert_called_once_with(2) + + @patch('swh.web.ui.views.api.service') @patch('swh.web.ui.views.api.request') @istest def api_uploadnsearch_bad_input(self, mock_request, mock_service): diff --git a/swh/web/ui/tests/views/test_browse.py b/swh/web/ui/tests/views/test_browse.py --- a/swh/web/ui/tests/views/test_browse.py +++ b/swh/web/ui/tests/views/test_browse.py @@ -567,9 +567,17 @@ mock_api.api_origin.assert_called_once_with(426) @patch('swh.web.ui.views.browse.api') + @patch('swh.web.ui.views.browse.url_for') @istest - def browse_origin_found(self, mock_api): + def browse_origin_found(self, mock_url_for, mock_api): # given + def url_for_test(fn, **args): + if fn == 'browse_revision_with_origin': + return '/browse/revision/origin/%s/' % args['origin_id'] + elif fn == 'api_origin_visits': + return '/api/1/stat/visits/%s/' % args['origin_id'] + mock_url_for.side_effect = url_for_test + mock_origin = {'type': 'git', 'lister': None, 'project': None, @@ -585,6 +593,10 @@ self.assert_template_used('origin.html') self.assertEqual(self.get_context_variable('origin_id'), 426) self.assertEqual(self.get_context_variable('origin'), mock_origin) + self.assertEqual(self.get_context_variable('browse_url'), + '/browse/revision/origin/426/') + self.assertEqual(self.get_context_variable('visit_url'), + '/api/1/stat/visits/426/') mock_api.api_origin.assert_called_once_with(426) diff --git a/swh/web/ui/views/api.py b/swh/web/ui/views/api.py --- a/swh/web/ui/views/api.py +++ b/swh/web/ui/views/api.py @@ -23,6 +23,19 @@ return service.stat_counters() +@app.route('/api/1/stat/visits//') +def api_origin_visits(origin_id): + """Return visit dates for the given revision. + + Returns: + A list of SWH visit occurrence timestamps, sorted from oldest to + newest. + + """ + date_gen = (item['date'] for item in service.stat_origin_visits(origin_id)) + return sorted(date_gen) + + @app.route('/api/1/search/', methods=['POST']) @app.route('/api/1/search//') def api_search(q=None): diff --git a/swh/web/ui/views/browse.py b/swh/web/ui/views/browse.py --- a/swh/web/ui/views/browse.py +++ b/swh/web/ui/views/browse.py @@ -246,7 +246,13 @@ """Browse origin with id id. """ - env = {'origin_id': origin_id, + + browse_url = url_for('browse_revision_with_origin', origin_id=origin_id) + visit_url = url_for('api_origin_visits', origin_id=origin_id) + + env = {'browse_url': browse_url, + 'visit_url': visit_url, + 'origin_id': origin_id, 'origin': None} try: